{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d8ecb6f3-6e26-45c5-a36e-5d63eda9324e",
   "metadata": {},
   "source": [
    "# Transformers 模型量化技术\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f8cbd7d1-4d97-4deb-a266-03228315ad99",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true,
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: transformers==4.36.0 in /usr/local/miniconda3/lib/python3.11/site-packages (4.36.0)\n",
      "Requirement already satisfied: filelock in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers==4.36.0) (3.13.1)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers==4.36.0) (0.19.4)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers==4.36.0) (1.26.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers==4.36.0) (23.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers==4.36.0) (6.0.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers==4.36.0) (2023.10.3)\n",
      "Requirement already satisfied: requests in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers==4.36.0) (2.31.0)\n",
      "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers==4.36.0) (0.15.2)\n",
      "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers==4.36.0) (0.4.0)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers==4.36.0) (4.65.0)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.36.0) (2023.10.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.36.0) (4.9.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->transformers==4.36.0) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->transformers==4.36.0) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->transformers==4.36.0) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->transformers==4.36.0) (2024.2.2)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: ffmpeg in /usr/local/miniconda3/lib/python3.11/site-packages (1.4)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: ffmpeg-python in /usr/local/miniconda3/lib/python3.11/site-packages (0.2.0)\n",
      "Requirement already satisfied: future in /usr/local/miniconda3/lib/python3.11/site-packages (from ffmpeg-python) (0.18.3)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: timm in /usr/local/miniconda3/lib/python3.11/site-packages (0.9.12)\n",
      "Requirement already satisfied: torch>=1.7 in /usr/local/miniconda3/lib/python3.11/site-packages (from timm) (2.2.0)\n",
      "Requirement already satisfied: torchvision in /usr/local/miniconda3/lib/python3.11/site-packages (from timm) (0.17.0)\n",
      "Requirement already satisfied: pyyaml in /usr/local/miniconda3/lib/python3.11/site-packages (from timm) (6.0.1)\n",
      "Requirement already satisfied: huggingface-hub in /usr/local/miniconda3/lib/python3.11/site-packages (from timm) (0.19.4)\n",
      "Requirement already satisfied: safetensors in /usr/local/miniconda3/lib/python3.11/site-packages (from timm) (0.4.0)\n",
      "Requirement already satisfied: filelock in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (3.13.1)\n",
      "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (4.9.0)\n",
      "Requirement already satisfied: sympy in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (1.12)\n",
      "Requirement already satisfied: networkx in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (3.2.1)\n",
      "Requirement already satisfied: jinja2 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (3.1.2)\n",
      "Requirement already satisfied: fsspec in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (2023.10.0)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (8.9.2.26)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (12.1.3.1)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (11.0.2.54)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (10.3.2.106)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (11.4.5.107)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (12.1.0.106)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (2.19.3)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (12.1.105)\n",
      "Requirement already satisfied: triton==2.2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.7->timm) (2.2.0)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/miniconda3/lib/python3.11/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.7->timm) (12.3.101)\n",
      "Requirement already satisfied: requests in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub->timm) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub->timm) (4.65.0)\n",
      "Requirement already satisfied: packaging>=20.9 in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub->timm) (23.2)\n",
      "Requirement already satisfied: numpy in /usr/local/miniconda3/lib/python3.11/site-packages (from torchvision->timm) (1.26.2)\n",
      "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from torchvision->timm) (10.1.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from jinja2->torch>=1.7->timm) (2.1.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->huggingface-hub->timm) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->huggingface-hub->timm) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->huggingface-hub->timm) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->huggingface-hub->timm) (2024.2.2)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/miniconda3/lib/python3.11/site-packages (from sympy->torch>=1.7->timm) (1.3.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: datasets in /usr/local/miniconda3/lib/python3.11/site-packages (2.17.0)\n",
      "Requirement already satisfied: filelock in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets) (3.13.1)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets) (1.26.2)\n",
      "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets) (14.0.1)\n",
      "Requirement already satisfied: pyarrow-hotfix in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets) (0.6)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets) (0.3.8)\n",
      "Requirement already satisfied: pandas in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets) (2.1.3)\n",
      "Requirement already satisfied: requests>=2.19.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets) (4.65.0)\n",
      "Requirement already satisfied: xxhash in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets) (3.4.1)\n",
      "Requirement already satisfied: multiprocess in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets) (0.70.16)\n",
      "Requirement already satisfied: fsspec<=2023.10.0,>=2023.1.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets) (2023.10.0)\n",
      "Requirement already satisfied: aiohttp in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets) (3.9.3)\n",
      "Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets) (0.19.4)\n",
      "Requirement already satisfied: packaging in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets) (23.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets) (6.0.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets) (23.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets) (1.4.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets) (1.9.3)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub>=0.19.4->datasets) (4.9.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (2024.2.2)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas->datasets) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas->datasets) (2023.3.post1)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas->datasets) (2023.3)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: evaluate in /usr/local/miniconda3/lib/python3.11/site-packages (0.4.1)\n",
      "Requirement already satisfied: datasets>=2.0.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from evaluate) (2.17.0)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from evaluate) (1.26.2)\n",
      "Requirement already satisfied: dill in /usr/local/miniconda3/lib/python3.11/site-packages (from evaluate) (0.3.8)\n",
      "Requirement already satisfied: pandas in /usr/local/miniconda3/lib/python3.11/site-packages (from evaluate) (2.1.3)\n",
      "Requirement already satisfied: requests>=2.19.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from evaluate) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from evaluate) (4.65.0)\n",
      "Requirement already satisfied: xxhash in /usr/local/miniconda3/lib/python3.11/site-packages (from evaluate) (3.4.1)\n",
      "Requirement already satisfied: multiprocess in /usr/local/miniconda3/lib/python3.11/site-packages (from evaluate) (0.70.16)\n",
      "Requirement already satisfied: fsspec>=2021.05.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from fsspec[http]>=2021.05.0->evaluate) (2023.10.0)\n",
      "Requirement already satisfied: huggingface-hub>=0.7.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from evaluate) (0.19.4)\n",
      "Requirement already satisfied: packaging in /usr/local/miniconda3/lib/python3.11/site-packages (from evaluate) (23.2)\n",
      "Requirement already satisfied: responses<0.19 in /usr/local/miniconda3/lib/python3.11/site-packages (from evaluate) (0.18.0)\n",
      "Requirement already satisfied: filelock in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets>=2.0.0->evaluate) (3.13.1)\n",
      "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets>=2.0.0->evaluate) (14.0.1)\n",
      "Requirement already satisfied: pyarrow-hotfix in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets>=2.0.0->evaluate) (0.6)\n",
      "Requirement already satisfied: aiohttp in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets>=2.0.0->evaluate) (3.9.3)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets>=2.0.0->evaluate) (6.0.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub>=0.7.0->evaluate) (4.9.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests>=2.19.0->evaluate) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests>=2.19.0->evaluate) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests>=2.19.0->evaluate) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests>=2.19.0->evaluate) (2024.2.2)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas->evaluate) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas->evaluate) (2023.3.post1)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas->evaluate) (2023.3)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (23.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.4.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.9.3)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->evaluate) (1.16.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: scikit-learn in /usr/local/miniconda3/lib/python3.11/site-packages (1.4.0)\n",
      "Requirement already satisfied: numpy<2.0,>=1.19.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from scikit-learn) (1.26.2)\n",
      "Requirement already satisfied: scipy>=1.6.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from scikit-learn) (1.12.0)\n",
      "Requirement already satisfied: joblib>=1.2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from scikit-learn) (1.3.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from scikit-learn) (3.3.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: pandas in /usr/local/miniconda3/lib/python3.11/site-packages (2.1.3)\n",
      "Requirement already satisfied: numpy<2,>=1.23.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas) (1.26.2)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas) (2023.3.post1)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas) (2023.3)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: peft in /usr/local/miniconda3/lib/python3.11/site-packages (0.8.2)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from peft) (1.26.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from peft) (23.2)\n",
      "Requirement already satisfied: psutil in /usr/local/miniconda3/lib/python3.11/site-packages (from peft) (5.9.6)\n",
      "Requirement already satisfied: pyyaml in /usr/local/miniconda3/lib/python3.11/site-packages (from peft) (6.0.1)\n",
      "Requirement already satisfied: torch>=1.13.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from peft) (2.2.0)\n",
      "Requirement already satisfied: transformers in /usr/local/miniconda3/lib/python3.11/site-packages (from peft) (4.36.0)\n",
      "Requirement already satisfied: tqdm in /usr/local/miniconda3/lib/python3.11/site-packages (from peft) (4.65.0)\n",
      "Requirement already satisfied: accelerate>=0.21.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from peft) (0.24.1)\n",
      "Requirement already satisfied: safetensors in /usr/local/miniconda3/lib/python3.11/site-packages (from peft) (0.4.0)\n",
      "Requirement already satisfied: huggingface-hub>=0.17.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from peft) (0.19.4)\n",
      "Requirement already satisfied: filelock in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub>=0.17.0->peft) (3.13.1)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub>=0.17.0->peft) (2023.10.0)\n",
      "Requirement already satisfied: requests in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub>=0.17.0->peft) (2.31.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub>=0.17.0->peft) (4.9.0)\n",
      "Requirement already satisfied: sympy in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->peft) (1.12)\n",
      "Requirement already satisfied: networkx in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->peft) (3.2.1)\n",
      "Requirement already satisfied: jinja2 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->peft) (3.1.2)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->peft) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->peft) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->peft) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->peft) (8.9.2.26)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->peft) (12.1.3.1)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->peft) (11.0.2.54)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->peft) (10.3.2.106)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->peft) (11.4.5.107)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->peft) (12.1.0.106)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->peft) (2.19.3)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->peft) (12.1.105)\n",
      "Requirement already satisfied: triton==2.2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->peft) (2.2.0)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/miniconda3/lib/python3.11/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.13.0->peft) (12.3.101)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers->peft) (2023.10.3)\n",
      "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers->peft) (0.15.2)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from jinja2->torch>=1.13.0->peft) (2.1.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->huggingface-hub>=0.17.0->peft) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->huggingface-hub>=0.17.0->peft) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->huggingface-hub>=0.17.0->peft) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->huggingface-hub>=0.17.0->peft) (2024.2.2)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/miniconda3/lib/python3.11/site-packages (from sympy->torch>=1.13.0->peft) (1.3.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: accelerate in /usr/local/miniconda3/lib/python3.11/site-packages (0.24.1)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from accelerate) (1.26.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from accelerate) (23.2)\n",
      "Requirement already satisfied: psutil in /usr/local/miniconda3/lib/python3.11/site-packages (from accelerate) (5.9.6)\n",
      "Requirement already satisfied: pyyaml in /usr/local/miniconda3/lib/python3.11/site-packages (from accelerate) (6.0.1)\n",
      "Requirement already satisfied: torch>=1.10.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from accelerate) (2.2.0)\n",
      "Requirement already satisfied: huggingface-hub in /usr/local/miniconda3/lib/python3.11/site-packages (from accelerate) (0.19.4)\n",
      "Requirement already satisfied: filelock in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (3.13.1)\n",
      "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (4.9.0)\n",
      "Requirement already satisfied: sympy in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (1.12)\n",
      "Requirement already satisfied: networkx in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (3.2.1)\n",
      "Requirement already satisfied: jinja2 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (3.1.2)\n",
      "Requirement already satisfied: fsspec in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (2023.10.0)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (8.9.2.26)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (12.1.3.1)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (11.0.2.54)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (10.3.2.106)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (11.4.5.107)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (12.1.0.106)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (2.19.3)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
      "Requirement already satisfied: triton==2.2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (2.2.0)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/miniconda3/lib/python3.11/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.10.0->accelerate) (12.3.101)\n",
      "Requirement already satisfied: requests in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub->accelerate) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub->accelerate) (4.65.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->huggingface-hub->accelerate) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->huggingface-hub->accelerate) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->huggingface-hub->accelerate) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->huggingface-hub->accelerate) (2024.2.2)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/miniconda3/lib/python3.11/site-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: autoawq in /usr/local/miniconda3/lib/python3.11/site-packages (0.2.0)\n",
      "Requirement already satisfied: torch>=2.0.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from autoawq) (2.2.0)\n",
      "Requirement already satisfied: transformers>=4.35.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from autoawq) (4.36.0)\n",
      "Requirement already satisfied: tokenizers>=0.12.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from autoawq) (0.15.2)\n",
      "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from autoawq) (4.9.0)\n",
      "Requirement already satisfied: accelerate in /usr/local/miniconda3/lib/python3.11/site-packages (from autoawq) (0.24.1)\n",
      "Requirement already satisfied: datasets in /usr/local/miniconda3/lib/python3.11/site-packages (from autoawq) (2.17.0)\n",
      "Requirement already satisfied: zstandard in /usr/local/miniconda3/lib/python3.11/site-packages (from autoawq) (0.19.0)\n",
      "Requirement already satisfied: autoawq-kernels in /usr/local/miniconda3/lib/python3.11/site-packages (from autoawq) (0.0.4)\n",
      "Requirement already satisfied: huggingface_hub<1.0,>=0.16.4 in /usr/local/miniconda3/lib/python3.11/site-packages (from tokenizers>=0.12.1->autoawq) (0.19.4)\n",
      "Requirement already satisfied: filelock in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=2.0.1->autoawq) (3.13.1)\n",
      "Requirement already satisfied: sympy in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=2.0.1->autoawq) (1.12)\n",
      "Requirement already satisfied: networkx in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=2.0.1->autoawq) (3.2.1)\n",
      "Requirement already satisfied: jinja2 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=2.0.1->autoawq) (3.1.2)\n",
      "Requirement already satisfied: fsspec in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=2.0.1->autoawq) (2023.10.0)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=2.0.1->autoawq) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=2.0.1->autoawq) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=2.0.1->autoawq) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=2.0.1->autoawq) (8.9.2.26)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=2.0.1->autoawq) (12.1.3.1)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=2.0.1->autoawq) (11.0.2.54)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=2.0.1->autoawq) (10.3.2.106)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=2.0.1->autoawq) (11.4.5.107)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=2.0.1->autoawq) (12.1.0.106)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=2.0.1->autoawq) (2.19.3)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=2.0.1->autoawq) (12.1.105)\n",
      "Requirement already satisfied: triton==2.2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=2.0.1->autoawq) (2.2.0)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/miniconda3/lib/python3.11/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=2.0.1->autoawq) (12.3.101)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers>=4.35.0->autoawq) (1.26.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers>=4.35.0->autoawq) (23.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers>=4.35.0->autoawq) (6.0.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers>=4.35.0->autoawq) (2023.10.3)\n",
      "Requirement already satisfied: requests in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers>=4.35.0->autoawq) (2.31.0)\n",
      "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers>=4.35.0->autoawq) (0.4.0)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers>=4.35.0->autoawq) (4.65.0)\n",
      "Requirement already satisfied: psutil in /usr/local/miniconda3/lib/python3.11/site-packages (from accelerate->autoawq) (5.9.6)\n",
      "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->autoawq) (14.0.1)\n",
      "Requirement already satisfied: pyarrow-hotfix in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->autoawq) (0.6)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->autoawq) (0.3.8)\n",
      "Requirement already satisfied: pandas in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->autoawq) (2.1.3)\n",
      "Requirement already satisfied: xxhash in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->autoawq) (3.4.1)\n",
      "Requirement already satisfied: multiprocess in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->autoawq) (0.70.16)\n",
      "Requirement already satisfied: aiohttp in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->autoawq) (3.9.3)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets->autoawq) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets->autoawq) (23.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets->autoawq) (1.4.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets->autoawq) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets->autoawq) (1.9.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->transformers>=4.35.0->autoawq) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->transformers>=4.35.0->autoawq) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->transformers>=4.35.0->autoawq) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->transformers>=4.35.0->autoawq) (2024.2.2)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from jinja2->torch>=2.0.1->autoawq) (2.1.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas->datasets->autoawq) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas->datasets->autoawq) (2023.3.post1)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas->datasets->autoawq) (2023.3)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/miniconda3/lib/python3.11/site-packages (from sympy->torch>=2.0.1->autoawq) (1.3.0)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets->autoawq) (1.16.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: optimum in /usr/local/miniconda3/lib/python3.11/site-packages (1.16.2)\n",
      "Requirement already satisfied: coloredlogs in /usr/local/miniconda3/lib/python3.11/site-packages (from optimum) (15.0.1)\n",
      "Requirement already satisfied: sympy in /usr/local/miniconda3/lib/python3.11/site-packages (from optimum) (1.12)\n",
      "Requirement already satisfied: transformers>=4.26.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers[sentencepiece]>=4.26.0->optimum) (4.36.0)\n",
      "Requirement already satisfied: torch>=1.11 in /usr/local/miniconda3/lib/python3.11/site-packages (from optimum) (2.2.0)\n",
      "Requirement already satisfied: packaging in /usr/local/miniconda3/lib/python3.11/site-packages (from optimum) (23.2)\n",
      "Requirement already satisfied: numpy in /usr/local/miniconda3/lib/python3.11/site-packages (from optimum) (1.26.2)\n",
      "Requirement already satisfied: huggingface-hub>=0.8.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from optimum) (0.19.4)\n",
      "Requirement already satisfied: datasets in /usr/local/miniconda3/lib/python3.11/site-packages (from optimum) (2.17.0)\n",
      "Requirement already satisfied: filelock in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub>=0.8.0->optimum) (3.13.1)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub>=0.8.0->optimum) (2023.10.0)\n",
      "Requirement already satisfied: requests in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub>=0.8.0->optimum) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub>=0.8.0->optimum) (4.65.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub>=0.8.0->optimum) (6.0.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub>=0.8.0->optimum) (4.9.0)\n",
      "Requirement already satisfied: networkx in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.11->optimum) (3.2.1)\n",
      "Requirement already satisfied: jinja2 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.11->optimum) (3.1.2)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.11->optimum) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.11->optimum) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.11->optimum) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.11->optimum) (8.9.2.26)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.11->optimum) (12.1.3.1)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.11->optimum) (11.0.2.54)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.11->optimum) (10.3.2.106)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.11->optimum) (11.4.5.107)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.11->optimum) (12.1.0.106)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.11->optimum) (2.19.3)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.11->optimum) (12.1.105)\n",
      "Requirement already satisfied: triton==2.2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.11->optimum) (2.2.0)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/miniconda3/lib/python3.11/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.11->optimum) (12.3.101)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers>=4.26.0->transformers[sentencepiece]>=4.26.0->optimum) (2023.10.3)\n",
      "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers>=4.26.0->transformers[sentencepiece]>=4.26.0->optimum) (0.15.2)\n",
      "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers>=4.26.0->transformers[sentencepiece]>=4.26.0->optimum) (0.4.0)\n",
      "Requirement already satisfied: sentencepiece!=0.1.92,>=0.1.91 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers[sentencepiece]>=4.26.0->optimum) (0.1.99)\n",
      "Requirement already satisfied: protobuf in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers[sentencepiece]>=4.26.0->optimum) (4.25.1)\n",
      "Requirement already satisfied: humanfriendly>=9.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from coloredlogs->optimum) (10.0)\n",
      "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->optimum) (14.0.1)\n",
      "Requirement already satisfied: pyarrow-hotfix in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->optimum) (0.6)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->optimum) (0.3.8)\n",
      "Requirement already satisfied: pandas in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->optimum) (2.1.3)\n",
      "Requirement already satisfied: xxhash in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->optimum) (3.4.1)\n",
      "Requirement already satisfied: multiprocess in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->optimum) (0.70.16)\n",
      "Requirement already satisfied: aiohttp in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->optimum) (3.9.3)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/miniconda3/lib/python3.11/site-packages (from sympy->optimum) (1.3.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets->optimum) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets->optimum) (23.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets->optimum) (1.4.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets->optimum) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets->optimum) (1.9.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->huggingface-hub>=0.8.0->optimum) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->huggingface-hub>=0.8.0->optimum) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->huggingface-hub>=0.8.0->optimum) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->huggingface-hub>=0.8.0->optimum) (2024.2.2)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from jinja2->torch>=1.11->optimum) (2.1.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas->datasets->optimum) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas->datasets->optimum) (2023.3.post1)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas->datasets->optimum) (2023.3)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets->optimum) (1.16.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: auto-gptq in /usr/local/miniconda3/lib/python3.11/site-packages (0.6.0)\n",
      "Requirement already satisfied: accelerate>=0.22.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from auto-gptq) (0.24.1)\n",
      "Requirement already satisfied: datasets in /usr/local/miniconda3/lib/python3.11/site-packages (from auto-gptq) (2.17.0)\n",
      "Requirement already satisfied: sentencepiece in /usr/local/miniconda3/lib/python3.11/site-packages (from auto-gptq) (0.1.99)\n",
      "Requirement already satisfied: numpy in /usr/local/miniconda3/lib/python3.11/site-packages (from auto-gptq) (1.26.2)\n",
      "Requirement already satisfied: rouge in /usr/local/miniconda3/lib/python3.11/site-packages (from auto-gptq) (1.0.1)\n",
      "Requirement already satisfied: gekko in /usr/local/miniconda3/lib/python3.11/site-packages (from auto-gptq) (1.0.6)\n",
      "Requirement already satisfied: torch>=1.13.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from auto-gptq) (2.2.0)\n",
      "Requirement already satisfied: safetensors in /usr/local/miniconda3/lib/python3.11/site-packages (from auto-gptq) (0.4.0)\n",
      "Requirement already satisfied: transformers>=4.31.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from auto-gptq) (4.36.0)\n",
      "Requirement already satisfied: peft>=0.5.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from auto-gptq) (0.8.2)\n",
      "Requirement already satisfied: tqdm in /usr/local/miniconda3/lib/python3.11/site-packages (from auto-gptq) (4.65.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from accelerate>=0.22.0->auto-gptq) (23.2)\n",
      "Requirement already satisfied: psutil in /usr/local/miniconda3/lib/python3.11/site-packages (from accelerate>=0.22.0->auto-gptq) (5.9.6)\n",
      "Requirement already satisfied: pyyaml in /usr/local/miniconda3/lib/python3.11/site-packages (from accelerate>=0.22.0->auto-gptq) (6.0.1)\n",
      "Requirement already satisfied: huggingface-hub in /usr/local/miniconda3/lib/python3.11/site-packages (from accelerate>=0.22.0->auto-gptq) (0.19.4)\n",
      "Requirement already satisfied: filelock in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (3.13.1)\n",
      "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (4.9.0)\n",
      "Requirement already satisfied: sympy in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (1.12)\n",
      "Requirement already satisfied: networkx in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (3.2.1)\n",
      "Requirement already satisfied: jinja2 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (3.1.2)\n",
      "Requirement already satisfied: fsspec in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (2023.10.0)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (8.9.2.26)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (12.1.3.1)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (11.0.2.54)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (10.3.2.106)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (11.4.5.107)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (12.1.0.106)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (2.19.3)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (12.1.105)\n",
      "Requirement already satisfied: triton==2.2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from torch>=1.13.0->auto-gptq) (2.2.0)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/miniconda3/lib/python3.11/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.13.0->auto-gptq) (12.3.101)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers>=4.31.0->auto-gptq) (2023.10.3)\n",
      "Requirement already satisfied: requests in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers>=4.31.0->auto-gptq) (2.31.0)\n",
      "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/miniconda3/lib/python3.11/site-packages (from transformers>=4.31.0->auto-gptq) (0.15.2)\n",
      "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->auto-gptq) (14.0.1)\n",
      "Requirement already satisfied: pyarrow-hotfix in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->auto-gptq) (0.6)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->auto-gptq) (0.3.8)\n",
      "Requirement already satisfied: pandas in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->auto-gptq) (2.1.3)\n",
      "Requirement already satisfied: xxhash in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->auto-gptq) (3.4.1)\n",
      "Requirement already satisfied: multiprocess in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->auto-gptq) (0.70.16)\n",
      "Requirement already satisfied: aiohttp in /usr/local/miniconda3/lib/python3.11/site-packages (from datasets->auto-gptq) (3.9.3)\n",
      "Requirement already satisfied: six in /usr/local/miniconda3/lib/python3.11/site-packages (from rouge->auto-gptq) (1.16.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets->auto-gptq) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets->auto-gptq) (23.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets->auto-gptq) (1.4.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets->auto-gptq) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp->datasets->auto-gptq) (1.9.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->transformers>=4.31.0->auto-gptq) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->transformers>=4.31.0->auto-gptq) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->transformers>=4.31.0->auto-gptq) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests->transformers>=4.31.0->auto-gptq) (2024.2.2)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from jinja2->torch>=1.13.0->auto-gptq) (2.1.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas->datasets->auto-gptq) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas->datasets->auto-gptq) (2023.3.post1)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas->datasets->auto-gptq) (2023.3)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/miniconda3/lib/python3.11/site-packages (from sympy->torch>=1.13.0->auto-gptq) (1.3.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: jiwer in /usr/local/miniconda3/lib/python3.11/site-packages (3.0.3)\n",
      "Requirement already satisfied: click<9.0.0,>=8.1.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from jiwer) (8.1.7)\n",
      "Requirement already satisfied: rapidfuzz<4,>=3 in /usr/local/miniconda3/lib/python3.11/site-packages (from jiwer) (3.6.1)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: librosa in /usr/local/miniconda3/lib/python3.11/site-packages (0.10.1)\n",
      "Requirement already satisfied: audioread>=2.1.9 in /usr/local/miniconda3/lib/python3.11/site-packages (from librosa) (3.0.1)\n",
      "Requirement already satisfied: numpy!=1.22.0,!=1.22.1,!=1.22.2,>=1.20.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from librosa) (1.26.2)\n",
      "Requirement already satisfied: scipy>=1.2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from librosa) (1.12.0)\n",
      "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from librosa) (1.4.0)\n",
      "Requirement already satisfied: joblib>=0.14 in /usr/local/miniconda3/lib/python3.11/site-packages (from librosa) (1.3.2)\n",
      "Requirement already satisfied: decorator>=4.3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from librosa) (5.1.1)\n",
      "Requirement already satisfied: numba>=0.51.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from librosa) (0.59.0)\n",
      "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from librosa) (0.12.1)\n",
      "Requirement already satisfied: pooch>=1.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from librosa) (1.8.0)\n",
      "Requirement already satisfied: soxr>=0.3.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from librosa) (0.3.7)\n",
      "Requirement already satisfied: typing-extensions>=4.1.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from librosa) (4.9.0)\n",
      "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from librosa) (0.3)\n",
      "Requirement already satisfied: msgpack>=1.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from librosa) (1.0.7)\n",
      "Requirement already satisfied: llvmlite<0.43,>=0.42.0dev0 in /usr/local/miniconda3/lib/python3.11/site-packages (from numba>=0.51.0->librosa) (0.42.0)\n",
      "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from pooch>=1.0->librosa) (3.10.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from pooch>=1.0->librosa) (23.2)\n",
      "Requirement already satisfied: requests>=2.19.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from pooch>=1.0->librosa) (2.31.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from scikit-learn>=0.20.0->librosa) (3.3.0)\n",
      "Requirement already satisfied: cffi>=1.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from soundfile>=0.12.1->librosa) (1.15.1)\n",
      "Requirement already satisfied: pycparser in /usr/local/miniconda3/lib/python3.11/site-packages (from cffi>=1.0->soundfile>=0.12.1->librosa) (2.21)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests>=2.19.0->pooch>=1.0->librosa) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests>=2.19.0->pooch>=1.0->librosa) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests>=2.19.0->pooch>=1.0->librosa) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests>=2.19.0->pooch>=1.0->librosa) (2024.2.2)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: langchain in /usr/local/miniconda3/lib/python3.11/site-packages (0.1.7)\n",
      "Requirement already satisfied: PyYAML>=5.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from langchain) (6.0.1)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/miniconda3/lib/python3.11/site-packages (from langchain) (2.0.27)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from langchain) (3.9.3)\n",
      "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /usr/local/miniconda3/lib/python3.11/site-packages (from langchain) (0.6.4)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /usr/local/miniconda3/lib/python3.11/site-packages (from langchain) (1.33)\n",
      "Requirement already satisfied: langchain-community<0.1,>=0.0.20 in /usr/local/miniconda3/lib/python3.11/site-packages (from langchain) (0.0.20)\n",
      "Requirement already satisfied: langchain-core<0.2,>=0.1.22 in /usr/local/miniconda3/lib/python3.11/site-packages (from langchain) (0.1.23)\n",
      "Requirement already satisfied: langsmith<0.1,>=0.0.83 in /usr/local/miniconda3/lib/python3.11/site-packages (from langchain) (0.0.87)\n",
      "Requirement already satisfied: numpy<2,>=1 in /usr/local/miniconda3/lib/python3.11/site-packages (from langchain) (1.26.2)\n",
      "Requirement already satisfied: pydantic<3,>=1 in /usr/local/miniconda3/lib/python3.11/site-packages (from langchain) (2.6.1)\n",
      "Requirement already satisfied: requests<3,>=2 in /usr/local/miniconda3/lib/python3.11/site-packages (from langchain) (2.31.0)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from langchain) (8.2.3)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.3)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain) (3.20.2)\n",
      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain) (0.9.0)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/miniconda3/lib/python3.11/site-packages (from jsonpatch<2.0,>=1.33->langchain) (2.1)\n",
      "Requirement already satisfied: anyio<5,>=3 in /usr/local/miniconda3/lib/python3.11/site-packages (from langchain-core<0.2,>=0.1.22->langchain) (3.7.1)\n",
      "Requirement already satisfied: packaging<24.0,>=23.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from langchain-core<0.2,>=0.1.22->langchain) (23.2)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from pydantic<3,>=1->langchain) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.16.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from pydantic<3,>=1->langchain) (2.16.2)\n",
      "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from pydantic<3,>=1->langchain) (4.9.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests<3,>=2->langchain) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests<3,>=2->langchain) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests<3,>=2->langchain) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests<3,>=2->langchain) (2024.2.2)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from SQLAlchemy<3,>=1.4->langchain) (3.0.3)\n",
      "Requirement already satisfied: sniffio>=1.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from anyio<5,>=3->langchain-core<0.2,>=0.1.22->langchain) (1.3.0)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain) (1.0.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: gradio in /usr/local/miniconda3/lib/python3.11/site-packages (3.39.0)\n",
      "Requirement already satisfied: aiofiles<24.0,>=22.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (23.2.1)\n",
      "Requirement already satisfied: aiohttp~=3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (3.9.3)\n",
      "Requirement already satisfied: altair<6.0,>=4.2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (5.1.2)\n",
      "Requirement already satisfied: fastapi in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (0.95.1)\n",
      "Requirement already satisfied: ffmpy in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (0.3.1)\n",
      "Requirement already satisfied: gradio-client>=0.3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (0.7.0)\n",
      "Requirement already satisfied: httpx in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (0.25.1)\n",
      "Requirement already satisfied: huggingface-hub>=0.14.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (0.19.4)\n",
      "Requirement already satisfied: jinja2<4.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (3.1.2)\n",
      "Requirement already satisfied: markdown-it-py>=2.0.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from markdown-it-py[linkify]>=2.0.0->gradio) (2.2.0)\n",
      "Requirement already satisfied: markupsafe~=2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (2.1.3)\n",
      "Requirement already satisfied: matplotlib~=3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (3.8.2)\n",
      "Requirement already satisfied: mdit-py-plugins<=0.3.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (0.3.3)\n",
      "Requirement already satisfied: numpy~=1.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (1.26.2)\n",
      "Requirement already satisfied: orjson~=3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (3.9.10)\n",
      "Requirement already satisfied: packaging in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (23.2)\n",
      "Requirement already satisfied: pandas<3.0,>=1.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (2.1.3)\n",
      "Requirement already satisfied: pillow<11.0,>=8.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (10.1.0)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,<3.0.0,>=1.7.4 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (2.6.1)\n",
      "Requirement already satisfied: pydub in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (0.25.1)\n",
      "Requirement already satisfied: python-multipart in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (0.0.6)\n",
      "Requirement already satisfied: pyyaml<7.0,>=5.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (6.0.1)\n",
      "Requirement already satisfied: requests~=2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (2.31.0)\n",
      "Requirement already satisfied: semantic-version~=2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (2.10.0)\n",
      "Requirement already satisfied: typing-extensions~=4.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (4.9.0)\n",
      "Requirement already satisfied: uvicorn>=0.14.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (0.24.0.post1)\n",
      "Requirement already satisfied: websockets<12.0,>=10.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio) (11.0.3)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp~=3.0->gradio) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp~=3.0->gradio) (23.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp~=3.0->gradio) (1.4.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp~=3.0->gradio) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp~=3.0->gradio) (1.9.3)\n",
      "Requirement already satisfied: jsonschema>=3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from altair<6.0,>=4.2.0->gradio) (4.20.0)\n",
      "Requirement already satisfied: toolz in /usr/local/miniconda3/lib/python3.11/site-packages (from altair<6.0,>=4.2.0->gradio) (0.12.0)\n",
      "Requirement already satisfied: fsspec in /usr/local/miniconda3/lib/python3.11/site-packages (from gradio-client>=0.3.0->gradio) (2023.10.0)\n",
      "Requirement already satisfied: filelock in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub>=0.14.0->gradio) (3.13.1)\n",
      "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from huggingface-hub>=0.14.0->gradio) (4.65.0)\n",
      "Requirement already satisfied: mdurl~=0.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from markdown-it-py>=2.0.0->markdown-it-py[linkify]>=2.0.0->gradio) (0.1.2)\n",
      "Requirement already satisfied: linkify-it-py<3,>=1 in /usr/local/miniconda3/lib/python3.11/site-packages (from markdown-it-py[linkify]>=2.0.0->gradio) (2.0.2)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from matplotlib~=3.0->gradio) (1.2.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/miniconda3/lib/python3.11/site-packages (from matplotlib~=3.0->gradio) (0.12.1)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from matplotlib~=3.0->gradio) (4.45.0)\n",
      "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from matplotlib~=3.0->gradio) (1.4.5)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from matplotlib~=3.0->gradio) (3.1.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/miniconda3/lib/python3.11/site-packages (from matplotlib~=3.0->gradio) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas<3.0,>=1.0->gradio) (2023.3.post1)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from pandas<3.0,>=1.0->gradio) (2023.3)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,<3.0.0,>=1.7.4->gradio) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.16.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,<3.0.0,>=1.7.4->gradio) (2.16.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests~=2.0->gradio) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests~=2.0->gradio) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests~=2.0->gradio) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/miniconda3/lib/python3.11/site-packages (from requests~=2.0->gradio) (2024.2.2)\n",
      "Requirement already satisfied: click>=7.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from uvicorn>=0.14.0->gradio) (8.1.7)\n",
      "Requirement already satisfied: h11>=0.8 in /usr/local/miniconda3/lib/python3.11/site-packages (from uvicorn>=0.14.0->gradio) (0.14.0)\n",
      "Collecting pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,<3.0.0,>=1.7.4 (from gradio)\n",
      "  Downloading pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (150 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m150.2/150.2 kB\u001b[0m \u001b[31m1.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: starlette<0.27.0,>=0.26.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from fastapi->gradio) (0.26.1)\n",
      "Requirement already satisfied: anyio in /usr/local/miniconda3/lib/python3.11/site-packages (from httpx->gradio) (3.7.1)\n",
      "Requirement already satisfied: httpcore in /usr/local/miniconda3/lib/python3.11/site-packages (from httpx->gradio) (1.0.2)\n",
      "Requirement already satisfied: sniffio in /usr/local/miniconda3/lib/python3.11/site-packages (from httpx->gradio) (1.3.0)\n",
      "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/miniconda3/lib/python3.11/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (2023.11.1)\n",
      "Requirement already satisfied: referencing>=0.28.4 in /usr/local/miniconda3/lib/python3.11/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (0.31.0)\n",
      "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (0.13.1)\n",
      "Requirement already satisfied: uc-micro-py in /usr/local/miniconda3/lib/python3.11/site-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify]>=2.0.0->gradio) (1.0.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from python-dateutil>=2.7->matplotlib~=3.0->gradio) (1.16.0)\n",
      "Downloading pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: pydantic\n",
      "  Attempting uninstall: pydantic\n",
      "    Found existing installation: pydantic 2.6.1\n",
      "    Uninstalling pydantic-2.6.1:\n",
      "      Successfully uninstalled pydantic-2.6.1\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "instructor 0.5.2 requires pydantic<3.0.0,>=2.0.2, but you have pydantic 1.10.14 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed pydantic-1.10.14\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: tensorflow-probability==0.23.0 in /usr/local/miniconda3/lib/python3.11/site-packages (0.23.0)\n",
      "Requirement already satisfied: absl-py in /usr/local/miniconda3/lib/python3.11/site-packages (from tensorflow-probability==0.23.0) (2.1.0)\n",
      "Requirement already satisfied: six>=1.10.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from tensorflow-probability==0.23.0) (1.16.0)\n",
      "Requirement already satisfied: numpy>=1.13.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from tensorflow-probability==0.23.0) (1.26.2)\n",
      "Requirement already satisfied: decorator in /usr/local/miniconda3/lib/python3.11/site-packages (from tensorflow-probability==0.23.0) (5.1.1)\n",
      "Requirement already satisfied: cloudpickle>=1.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from tensorflow-probability==0.23.0) (3.0.0)\n",
      "Requirement already satisfied: gast>=0.3.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from tensorflow-probability==0.23.0) (0.5.4)\n",
      "Requirement already satisfied: dm-tree in /usr/local/miniconda3/lib/python3.11/site-packages (from tensorflow-probability==0.23.0) (0.1.8)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: instructor in /usr/local/miniconda3/lib/python3.11/site-packages (0.5.2)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.9.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from instructor) (3.9.3)\n",
      "Requirement already satisfied: docstring-parser<0.16,>=0.15 in /usr/local/miniconda3/lib/python3.11/site-packages (from instructor) (0.15)\n",
      "Requirement already satisfied: openai<2.0.0,>=1.1.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from instructor) (1.3.5)\n",
      "Collecting pydantic<3.0.0,>=2.0.2 (from instructor)\n",
      "  Using cached pydantic-2.6.1-py3-none-any.whl.metadata (83 kB)\n",
      "Requirement already satisfied: rich<14.0.0,>=13.7.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from instructor) (13.7.0)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.2.3 in /usr/local/miniconda3/lib/python3.11/site-packages (from instructor) (8.2.3)\n",
      "Requirement already satisfied: typer<0.10.0,>=0.9.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from instructor) (0.9.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.9.1->instructor) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.9.1->instructor) (23.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.9.1->instructor) (1.4.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.9.1->instructor) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.9.1->instructor) (1.9.3)\n",
      "Requirement already satisfied: anyio<4,>=3.5.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from openai<2.0.0,>=1.1.0->instructor) (3.7.1)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from openai<2.0.0,>=1.1.0->instructor) (1.8.0)\n",
      "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from openai<2.0.0,>=1.1.0->instructor) (0.25.1)\n",
      "Requirement already satisfied: tqdm>4 in /usr/local/miniconda3/lib/python3.11/site-packages (from openai<2.0.0,>=1.1.0->instructor) (4.65.0)\n",
      "Requirement already satisfied: typing-extensions<5,>=4.5 in /usr/local/miniconda3/lib/python3.11/site-packages (from openai<2.0.0,>=1.1.0->instructor) (4.9.0)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.2->instructor) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.16.2 in /usr/local/miniconda3/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.2->instructor) (2.16.2)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from rich<14.0.0,>=13.7.0->instructor) (2.2.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/miniconda3/lib/python3.11/site-packages (from rich<14.0.0,>=13.7.0->instructor) (2.17.2)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from typer<0.10.0,>=0.9.0->instructor) (8.1.7)\n",
      "Requirement already satisfied: idna>=2.8 in /usr/local/miniconda3/lib/python3.11/site-packages (from anyio<4,>=3.5.0->openai<2.0.0,>=1.1.0->instructor) (3.4)\n",
      "Requirement already satisfied: sniffio>=1.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from anyio<4,>=3.5.0->openai<2.0.0,>=1.1.0->instructor) (1.3.0)\n",
      "Requirement already satisfied: certifi in /usr/local/miniconda3/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.1.0->instructor) (2024.2.2)\n",
      "Requirement already satisfied: httpcore in /usr/local/miniconda3/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.1.0->instructor) (1.0.2)\n",
      "Requirement already satisfied: mdurl~=0.1 in /usr/local/miniconda3/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=13.7.0->instructor) (0.1.2)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/miniconda3/lib/python3.11/site-packages (from httpcore->httpx<1,>=0.23.0->openai<2.0.0,>=1.1.0->instructor) (0.14.0)\n",
      "Using cached pydantic-2.6.1-py3-none-any.whl (394 kB)\n",
      "Installing collected packages: pydantic\n",
      "  Attempting uninstall: pydantic\n",
      "    Found existing installation: pydantic 1.10.14\n",
      "    Uninstalling pydantic-1.10.14:\n",
      "      Successfully uninstalled pydantic-1.10.14\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "fastapi 0.95.1 requires pydantic!=1.7,!=1.7.1,!=1.7.2,!=1.7.3,!=1.8,!=1.8.1,<2.0.0,>=1.6.2, but you have pydantic 2.6.1 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed pydantic-2.6.1\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "### 环境依赖\n",
    "\n",
    "!pip install torch>=2.1.2\n",
    "!pip install transformers==4.36.0\n",
    "!pip install ffmpeg\n",
    "!pip install ffmpeg-python\n",
    "!pip install timm\n",
    "!pip install datasets\n",
    "!pip install evaluate\n",
    "!pip install scikit-learn\n",
    "!pip install pandas\n",
    "!pip install peft\n",
    "!pip install accelerate\n",
    "!pip install autoawq\n",
    "!pip install optimum\n",
    "!pip install auto-gptq\n",
    "!pip install bitsandbytes>0.39.0\n",
    "!pip install jiwer\n",
    "!pip install soundfile>=0.12.1\n",
    "!pip install librosa\n",
    "!pip install langchain\n",
    "!pip install gradio\n",
    "!pip install tensorflow-probability==0.23.0\n",
    "!pip install instructor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "69800eef-5f9a-4781-88c8-0a2cfab6d802",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fri Feb 16 11:34:02 2024       \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.2     |\n",
      "|-----------------------------------------+----------------------+----------------------+\n",
      "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                                         |                      |               MIG M. |\n",
      "|=========================================+======================+======================|\n",
      "|   0  NVIDIA GeForce RTX 4090        Off | 00000000:00:03.0 Off |                  Off |\n",
      "| 33%   41C    P0              N/A / 450W |      2MiB / 24564MiB |      0%      Default |\n",
      "|                                         |                      |                  N/A |\n",
      "+-----------------------------------------+----------------------+----------------------+\n",
      "                                                                                         \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| Processes:                                                                            |\n",
      "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
      "|        ID   ID                                                             Usage      |\n",
      "|=======================================================================================|\n",
      "|  No running processes found                                                           |\n",
      "+---------------------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a13a78b7-d128-49e9-b8e3-ca8cdc67af29",
   "metadata": {},
   "source": [
    "## 量化前模型测试文本生成任务"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "43ea82e3-3504-4b2e-aa98-434fab549395",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e8c40e9a86df4fdc919aeb51fa81be47",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/miniconda3/lib/python3.11/site-packages/transformers/generation/utils.py:1133: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[\"Hello, I'm am conscious and aware of my surroundings.\\nI'm not sure if you\"]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# huggingface.co How to use\n",
    "\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "import torch\n",
    "\n",
    "model_path = \"facebook/opt-6.7b\"\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(model_path, torch_dtype=torch.float16).cuda()\n",
    "\n",
    "# the fast tokenizer currently does not work correctly\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-6.7b\", use_fast=False)\n",
    "\n",
    "prompt = \"Hello, I'm am conscious and\"\n",
    "\n",
    "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids.cuda()\n",
    "\n",
    "generated_ids = model.generate(input_ids)\n",
    "\n",
    "tokenizer.batch_decode(generated_ids, skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "41024104-8a98-4737-a819-a8b31fdb4120",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fri Feb 16 12:39:16 2024       \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.2     |\n",
      "|-----------------------------------------+----------------------+----------------------+\n",
      "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                                         |                      |               MIG M. |\n",
      "|=========================================+======================+======================|\n",
      "|   0  NVIDIA GeForce RTX 4090        Off | 00000000:00:03.0 Off |                  Off |\n",
      "| 44%   42C    P8              10W / 450W |  13217MiB / 24564MiB |      0%      Default |\n",
      "|                                         |                      |                  N/A |\n",
      "+-----------------------------------------+----------------------+----------------------+\n",
      "                                                                                         \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| Processes:                                                                            |\n",
      "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
      "|        ID   ID                                                             Usage      |\n",
      "|=======================================================================================|\n",
      "|    0   N/A  N/A      8336      C   /usr/local/miniconda3/bin/python          13208MiB |\n",
      "+---------------------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "#### 实测GPU显存占用：加载 OPT-6.7B 模型后\n",
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6b50ca0a-992f-4196-b07f-1280b1349f7b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2118dae970de47ef8eeb085b66aee386",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 使用课程中的 Demo Code 会出现 OutOfMemoryError\n",
    "from transformers import pipeline\n",
    "import torch\n",
    "\n",
    "model_path = \"facebook/opt-6.7b\"\n",
    "\n",
    "# 使用 GPU 加载原始的 OPT-6.7B 模型\n",
    "generator = pipeline('text-generation',\n",
    "                     model=model_path,\n",
    "                     device=0,\n",
    "                     torch_dtype=torch.float16, # 引入torch包和使用torch_dtype参数 解决了 OutOfMemoryError\n",
    "                     do_sample=True,\n",
    "                     num_return_sequences=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "24b1404d-6c6e-42c3-a9be-7480ca1e6ba1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fri Feb 16 12:48:21 2024       \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.2     |\n",
      "|-----------------------------------------+----------------------+----------------------+\n",
      "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                                         |                      |               MIG M. |\n",
      "|=========================================+======================+======================|\n",
      "|   0  NVIDIA GeForce RTX 4090        Off | 00000000:00:03.0 Off |                  Off |\n",
      "| 33%   42C    P8              16W / 450W |  13263MiB / 24564MiB |      0%      Default |\n",
      "|                                         |                      |                  N/A |\n",
      "+-----------------------------------------+----------------------+----------------------+\n",
      "                                                                                         \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| Processes:                                                                            |\n",
      "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
      "|        ID   ID                                                             Usage      |\n",
      "|=======================================================================================|\n",
      "|    0   N/A  N/A      8983      C   /usr/local/miniconda3/bin/python          13254MiB |\n",
      "+---------------------------------------------------------------------------------------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "#### 实测GPU显存占用：加载 OPT-6.7B 模型后\n",
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6b40d5b2-d06a-49d4-aded-7d41d974e948",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'generated_text': 'The woman worked as a contractor, no? How could she have lost her job? This story isn'},\n",
       " {'generated_text': 'The woman worked as a receptionist for the hospital in Fife where the man was being treated.'},\n",
       " {'generated_text': 'The woman worked as a journalist in the UK for eight years, and has worked as a blogger covering'}]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generator(\"The woman worked as a\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "214bc4b8-0435-485e-a620-14f7a3b0271b",
   "metadata": {},
   "source": [
    "## 使用 AutoAWQ 量化模型\n",
    "\n",
    "下面我们以 `facebook opt-6.7B` 模型为例，使用 `AutoAWQ` 库实现的 AWQ 算法实现模型量化。\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "00148e06-04a5-4183-9725-210ad6ecbee4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from awq import AutoAWQForCausalLM\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "model_path = \"facebook/opt-6.7b\"\n",
    "quant_path = \"models/opt-6.7b-awq\"\n",
    "quant_config = {\"zero_point\": True, \"q_group_size\": 128, \"w_bit\": 4, \"version\": \"GEMM\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9dc9341f-489d-46c1-bb4c-1f73a3202dba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "253a1ccea8c440a496213c292a97c2f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fetching 9 files:   0%|          | 0/9 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae5da3fff40247bd9efbf88541ea6480",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 加载模型\n",
    "model = AutoAWQForCausalLM.from_pretrained(model_path)\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a5d9d6a7-f146-4235-bcfe-e957235b7dbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/miniconda3/lib/python3.11/site-packages/huggingface_hub/repocard.py:105: UserWarning: Repo card metadata block was not found. Setting CardData to empty.\n",
      "  warnings.warn(\"Repo card metadata block was not found. Setting CardData to empty.\")\n",
      "AWQ: 100%|██████████| 32/32 [09:04<00:00, 17.00s/it]\n"
     ]
    }
   ],
   "source": [
    "# 量化模型\n",
    "model.quantize(tokenizer, quant_config=quant_config)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "16e5ea4a-716d-4f0c-92cb-949f432cdc39",
   "metadata": {},
   "source": [
    "## 实测GPU显存占用：量化模型（峰值超过9GB）\n",
    "\n",
    "```shell\n",
    "nvidia-smi\n",
    "\n",
    "Every 2.0s: nvidia-smi                                                                                   10-60-14-133: Fri Feb 16 13:34:56 2024\n",
    "\n",
    "Fri Feb 16 13:34:56 2024\n",
    "+---------------------------------------------------------------------------------------+\n",
    "| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.2     |\n",
    "|-----------------------------------------+----------------------+----------------------+\n",
    "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
    "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
    "|                                         |                      |               MIG M. |\n",
    "|=========================================+======================+======================|\n",
    "|   0  NVIDIA GeForce RTX 4090        Off | 00000000:00:03.0 Off |                  Off |\n",
    "|100%   64C    P2             270W / 450W |   9199MiB / 24564MiB |     79%      Default |\n",
    "|                                         |                      |                  N/A |\n",
    "+-----------------------------------------+----------------------+----------------------+\n",
    "\n",
    "+---------------------------------------------------------------------------------------+\n",
    "| Processes:                                                                            |\n",
    "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
    "|        ID   ID                                                             Usage      |\n",
    "|=======================================================================================|\n",
    "|    0   N/A  N/A     11782      C   /usr/local/miniconda3/bin/python           9199MiB |\n",
    "+---------------------------------------------------------------------------------------+\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "047fa0b7-0fe2-45cb-8470-8ed81f93c720",
   "metadata": {},
   "source": [
    "## Transformers 兼容性配置\n",
    "\n",
    "为了使`quant_config` 与 transformers 兼容，我们需要修改配置文件：`使用 Transformers.AwqConfig 来实例化量化模型配置`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "407a9514-dab3-4c22-b3e8-fc3da7233427",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AwqConfig, AutoConfig\n",
    "\n",
    "# 修改配置文件以使其与transformers集成兼容\n",
    "quantization_config = AwqConfig(\n",
    "    bits=quant_config[\"w_bit\"],\n",
    "    group_size=quant_config[\"q_group_size\"],\n",
    "    zero_point=quant_config[\"zero_point\"],\n",
    "    version=quant_config[\"version\"].lower(),\n",
    ").to_dict()\n",
    "\n",
    "# 预训练的transformers模型存储在model属性中，我们需要传递一个字典\n",
    "model.model.config.quantization_config = quantization_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "28d1c8e8-f257-4431-9dcc-674c0fb72284",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('models/opt-6.7b-awq/tokenizer_config.json',\n",
       " 'models/opt-6.7b-awq/special_tokens_map.json',\n",
       " 'models/opt-6.7b-awq/vocab.json',\n",
       " 'models/opt-6.7b-awq/merges.txt',\n",
       " 'models/opt-6.7b-awq/added_tokens.json',\n",
       " 'models/opt-6.7b-awq/tokenizer.json')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 保存模型权重\n",
    "model.save_quantized(quant_path)\n",
    "tokenizer.save_pretrained(quant_path)  # 保存分词器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6d8d262f-ed92-4715-9893-52128aec6ba5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OptAWQForCausalLM(\n",
       "  (model): OPTForCausalLM(\n",
       "    (model): OPTModel(\n",
       "      (decoder): OPTDecoder(\n",
       "        (embed_tokens): Embedding(50272, 4096, padding_idx=1)\n",
       "        (embed_positions): OPTLearnedPositionalEmbedding(2050, 4096)\n",
       "        (final_layer_norm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
       "        (layers): ModuleList(\n",
       "          (0-31): 32 x OPTDecoderLayer(\n",
       "            (self_attn): OPTAttention(\n",
       "              (k_proj): WQLinear_GEMM(in_features=4096, out_features=4096, bias=True, w_bit=4, group_size=128)\n",
       "              (v_proj): WQLinear_GEMM(in_features=4096, out_features=4096, bias=True, w_bit=4, group_size=128)\n",
       "              (q_proj): WQLinear_GEMM(in_features=4096, out_features=4096, bias=True, w_bit=4, group_size=128)\n",
       "              (out_proj): WQLinear_GEMM(in_features=4096, out_features=4096, bias=True, w_bit=4, group_size=128)\n",
       "            )\n",
       "            (activation_fn): ReLU()\n",
       "            (self_attn_layer_norm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): WQLinear_GEMM(in_features=4096, out_features=16384, bias=True, w_bit=4, group_size=128)\n",
       "            (fc2): WQLinear_GEMM(in_features=16384, out_features=4096, bias=True, w_bit=4, group_size=128)\n",
       "            (final_layer_norm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (lm_head): Linear(in_features=4096, out_features=50272, bias=False)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 模型评估\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0c6275d-976c-4304-b834-827fdc89870e",
   "metadata": {},
   "source": [
    "### 使用 GPU 加载 AWQ 量化后的6.7B模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b930ed95-40a7-4c68-9da6-351f83f477d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(quant_path)\n",
    "model = AutoModelForCausalLM.from_pretrained(quant_path, device_map=\"cuda\").to(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "71e1d5e7-9dee-4a45-aec7-9234dd90aeba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 文本生成函数\n",
    "def generate_text(text):\n",
    "    inputs = tokenizer(text, return_tensors=\"pt\").to(0)\n",
    "\n",
    "    out = model.generate(**inputs, max_new_tokens=64)\n",
    "    return tokenizer.decode(out[0], skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5d4b93f9-e899-441b-8a44-2906c0e6fad5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Happy New Year! I'm glad to finally be here and be here and be here and be here and be here and be here and be here and be here and be here and be here\n",
      " be here and be here and be here and be here and be here and be here and be here and be here and be here and be here and be here and\n"
     ]
    }
   ],
   "source": [
    "result = generate_text(\"Happy New Year! I'm glad to\")\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a9518ed7-256a-4337-b067-2e37cd1730fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "新春快乐 龙年大吉 龙龝 龰龞 龰龝 龰龜 龰龜 龰龜 龰龜 龰� 龰龜 龰龜 龱龜 龰龜 龰龿\n"
     ]
    }
   ],
   "source": [
    "## 中文🤔\n",
    "result = generate_text(\"新春快乐 龙年大吉\")\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ee9ce463-a3ed-46ea-9290-6a24b1e6db0a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Opportunity knocks but once, You can change but it is you can\n",
      "\n",
      "This you can change, it is you can call, You can change but it is you can change, you can change but it is you can change     Opportun opportunity opportunity opportunity, you can change but it is you want can change it is you can change it\n"
     ]
    }
   ],
   "source": [
    "result =  generate_text(\"Opportunity knocks but once, You can\")\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "346d1d2e-6955-4048-a8aa-73fb6107d5e2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The woman worked as a secretary in a school and a house and a secretary in a office and a secretary is a secretary\n",
      " a secretary in a a with a secretary in a a school is a secretary in a school and a secretary in a a a secretary in a a a the with a secretary in a a company and a secretary in a a\n"
     ]
    }
   ],
   "source": [
    "result = generate_text(\"The woman worked as a\")\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c23a37ed-7681-45a2-af66-1eff7019f124",
   "metadata": {},
   "source": [
    "# Transformers 模型量化技术：GPTQ\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9444cf57-fdaa-4831-b49d-a76ef7a1d462",
   "metadata": {},
   "outputs": [],
   "source": [
    "## 使用 GPTQ 量化 OPT-6.7B 模型\n",
    "\n",
    "model_id = \"facebook/opt-6.7b\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2a52451e-f72f-486d-9d73-b0eaf4a5cc01",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "CUDA extension not installed.\n",
      "CUDA extension not installed.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45b04c4f92eb4a78b27f4062e398172a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1b84d7b68304928affc9a71b60251c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading readme:   0%|          | 0.00/10.5k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b074fc4647554b918c9610726141ba45",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/733k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af392773d19e44c9b1a1c5e8e0aad393",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/6.36M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93de0d9caf1f488b9cb36242c9b4cae6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/657k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd3f914b5fc142edacf0216a792090d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split:   0%|          | 0/4358 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02bd03822c7a4b65ba54793a5b08d114",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split:   0%|          | 0/36718 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "72bfdce402654a308678770b3adbeb9a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split:   0%|          | 0/3760 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c25320c4c9fb47dc83aedb36798a7938",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing model.decoder.layers blocks :   0%|          | 0/32 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer, GPTQConfig\n",
    "import torch\n",
    "\n",
    "quantization_config = GPTQConfig(\n",
    "     bits=4, # 量化精度\n",
    "     group_size=128,\n",
    "     dataset=\"wikitext2\",\n",
    "     desc_act=False,\n",
    ")\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
    "model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=quantization_config, device_map='auto')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c18bba4-a95c-421c-93e7-d08fcaf1bf25",
   "metadata": {},
   "source": [
    "#### 检查量化模型正确性\n",
    "\n",
    "通过检查`线性层的属性`来确保模型已正确量化，它们应该包含`qweight`和`qzeros`属性，这些属性应该是`torch.int32`数据类型。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "60339c47-0426-4523-a415-f543b4c5b153",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'training': True,\n",
       " '_parameters': OrderedDict(),\n",
       " '_buffers': OrderedDict([('qweight',\n",
       "               tensor([[-1516590485,  2022282410, -1182414405,  ..., -1537760937,\n",
       "                          968190105, -1218869078],\n",
       "                       [-1973762678, -1990888091,  1451669112,  ...,  1767327094,\n",
       "                         1485273242, -1769109111],\n",
       "                       [ -890521657,  1705355194,  2042256023,  ...,  1401453177,\n",
       "                         -963081656, -1212573545],\n",
       "                       ...,\n",
       "                       [-1192544404,   713964148,  1432856439,  ...,  1967750874,\n",
       "                        -1214732472, -1787328359],\n",
       "                       [-1485273703, -1736799852,  1776327546,  ..., -1757500853,\n",
       "                        -1936219177, -1986435164],\n",
       "                       [ 1817626772,  1151063749, -1735997610,  ..., -1334205594,\n",
       "                        -1450811000, -1199921061]], device='cuda:0', dtype=torch.int32)),\n",
       "              ('qzeros',\n",
       "               tensor([[2004318071, 2004318071, 2004318071,  ..., 2004318071, 2004318071,\n",
       "                        2004318071],\n",
       "                       [2004318071, 2004318071, 2004318071,  ..., 2004318071, 2004318071,\n",
       "                        2004318071],\n",
       "                       [2004318071, 2004318071, 2004318071,  ..., 2004318071, 2004318071,\n",
       "                        2004318071],\n",
       "                       ...,\n",
       "                       [2004318071, 2004318071, 2004318071,  ..., 2004318071, 2004318071,\n",
       "                        2004318071],\n",
       "                       [2004318071, 2004318071, 2004318071,  ..., 2004318071, 2004318071,\n",
       "                        2004318071],\n",
       "                       [2004318071, 2004318071, 2004318071,  ..., 2004318071, 2004318071,\n",
       "                        2004318071]], device='cuda:0', dtype=torch.int32)),\n",
       "              ('scales',\n",
       "               tensor([[0.0039, 0.0035, 0.0035,  ..., 0.0030, 0.0028, 0.0044],\n",
       "                       [0.0032, 0.0030, 0.0036,  ..., 0.0034, 0.0028, 0.0033],\n",
       "                       [0.0038, 0.0029, 0.0039,  ..., 0.0043, 0.0024, 0.0035],\n",
       "                       ...,\n",
       "                       [0.0034, 0.0034, 0.0034,  ..., 0.0033, 0.0038, 0.0030],\n",
       "                       [0.0030, 0.0029, 0.0030,  ..., 0.0031, 0.0038, 0.0027],\n",
       "                       [0.0034, 0.0035, 0.0032,  ..., 0.0026, 0.0027, 0.0041]],\n",
       "                      device='cuda:0', dtype=torch.float16)),\n",
       "              ('g_idx',\n",
       "               tensor([ 0,  0,  0,  ..., 31, 31, 31], device='cuda:0', dtype=torch.int32)),\n",
       "              ('bias',\n",
       "               tensor([ 0.0061,  0.0129, -0.0215,  ...,  0.0157, -0.0144,  0.0106],\n",
       "                      device='cuda:0', dtype=torch.float16))]),\n",
       " '_non_persistent_buffers_set': set(),\n",
       " '_backward_pre_hooks': OrderedDict(),\n",
       " '_backward_hooks': OrderedDict(),\n",
       " '_is_full_backward_hook': None,\n",
       " '_forward_hooks': OrderedDict(),\n",
       " '_forward_hooks_with_kwargs': OrderedDict(),\n",
       " '_forward_hooks_always_called': OrderedDict(),\n",
       " '_forward_pre_hooks': OrderedDict(),\n",
       " '_forward_pre_hooks_with_kwargs': OrderedDict(),\n",
       " '_state_dict_hooks': OrderedDict(),\n",
       " '_state_dict_pre_hooks': OrderedDict(),\n",
       " '_load_state_dict_pre_hooks': OrderedDict(),\n",
       " '_load_state_dict_post_hooks': OrderedDict(),\n",
       " '_modules': OrderedDict(),\n",
       " 'infeatures': 4096,\n",
       " 'outfeatures': 4096,\n",
       " 'bits': 4,\n",
       " 'group_size': 128,\n",
       " 'maxq': 15,\n",
       " 'half_indim': 2048,\n",
       " 'use_cuda_fp16': True,\n",
       " 'wf': tensor([[ 0,  4,  8, 12, 16, 20, 24, 28]], dtype=torch.int32),\n",
       " 'kernel_switch_threshold': 128,\n",
       " 'autogptq_cuda_available': False,\n",
       " 'autogptq_cuda': None,\n",
       " 'trainable': False,\n",
       " 'device': device(type='cuda', index=0)}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.model.decoder.layers[0].self_attn.q_proj.__dict__"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0091d39b-67fd-45da-b1cb-497ac4e2a879",
   "metadata": {},
   "source": [
    "#### 使用 GPU 加载模型并生成文本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "13d03ce3-385e-44bb-930e-2fd3e2e91930",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Happy New Year! I'm glad to see you're still around.\n",
      "I'm still around, just not as active as I used to be.\n"
     ]
    }
   ],
   "source": [
    "text = \"Happy New Year! I'm glad to\"\n",
    "inputs = tokenizer(text, return_tensors=\"pt\").to(0)\n",
    "\n",
    "out = model.generate(**inputs, max_new_tokens=64)\n",
    "print(tokenizer.decode(out[0], skip_special_tokens=True))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
